package com.whoami.idmagic.mllib

import org.apache.spark.sql.SparkSession

import scala.collection.mutable


/**
  *
  * @author tzp
  * @since 2019/10/28
  */
object appGender {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName("apprank")
      .getOrCreate()
    val sc = spark.sparkContext
    import spark.implicits._

    val a = mutable.ListBuffer[Map[String, String]]()
    a += Map("package" -> "app.soyoung.com", "name" -> "sy")
    a += Map("package" -> "app.baidu.com", "name" -> "bd")
    a += Map("package" -> "app.qq.com", "name" -> "qq")
    a += Map("package" -> "app.taobao.com", "name" -> "tb")
    a += Map("package" -> "app.jd.com", "name" -> "jd")

    val b = Array(("imei1", a), ("imei2", a))
    val data = sc.parallelize(b)

    data.toDF().write.orc("/user/tzp/mllib/sample1")

    val x = spark.read.orc("/user/tzp/mllib/sample1")
    val y = x.map(f => {
      val al = f.getSeq[Map[String, String]](1)
      var pa = al.map(a => {
        a.get("package")
      }).filter(a => {
        a.nonEmpty && a.get != ""
      }).toArray

      (f.getString(0), pa)
    })
    y.toDF("imei", "packlist").write.orc("/user/tzp/mllib/sample2")

    val z = spark.read.orc("/user/tzp/mllib/sample2")

    val cjd = z.filter(a => {
      var rs = false
      var packages = a.getSeq[String](1)
      for (pa <- packages) {
        if (pa.contains("jd")) {
          rs = true
        }
      }
      rs
    })
  }
}
